#' 提取结局的数据
#'
#' @param exposure_iv 工具变量的数据框，不只是SNP
#' @param outcome 结局，3个选择：1.单个或多个openGWAS ID；
#'  2.单个数据框，U1_Clean_data清洗过后的df;
#'  3.单个parquet文件路径（例如"F:/OneclickDatabase/Oneclick_GWAS_ID/Oneclick_a_0001.parquet"）
#'    或多个parquet文件路径（例如c("F:/OneclickDatabase/Oneclick_GWAS_ID/Oneclick_a_0001.parquet",
#'                                 "F:/OneclickDatabase/Oneclick_GWAS_ID/Oneclick_a_0002.parquet")）
#'
#' @param proxies 是或否，以0.8的r2阈值找代理
#' @param max_retries 是重试计算器，默认是10，不能超过50，这个参数是对Server code: 502的一种解决办法
#' @param pop 参考U2_extract_instruments的说明
#' @param bfile 参考U2_extract_instruments的说明
#' @param plink_bin 参考U2_extract_instruments的说明
#'
#' @return 结局数据
#' @export
#'
#' @examples
#'
#' # 类似提取在线ID，寻找代理的代码修改自PMID:32996171
#' \dontrun{
#'
#' # 这种情况可以合并使用，但是需要是list
#'  outcome<-c("ebi-a-GCST90020053",
#'   "finngen_R10_M13_DISCINFECTION.parquet") %>%
#'     split( list(.) )
#'
#' outs <- U3_extract_outcomes_data(outcome)
#'
#'   }
#'
U3_extract_outcomes_data <- function(  outcome =outcome,
                                       exposure_iv = exposure_iv,
                                       proxies=FALSE,
                                       max_retries = 10,
                                       pop = "EUR",
                                       bfile = "",
                                       plink_bin = "" ){

  stopifnot(max_retries <= 100)

  if(!proxies){
    message("默认不找代理SNP

##############################################################################
    对应的论文部分（降重后使用）：

    no Proxy SNPs: In instances Where SNPs were not available in the outcome GWAS, no proxy SNPs were used.
##############################################################################

            ")
  }else{

    message("
##############################################################################
    对应的论文部分（降重后使用）：

    Proxy SNPs: Where SNPs were not available in the outcome GWAS, the ",pop," thousand genomes was queried to identify potential proxy SNPs that are in linkage disequilibrium (r2 > 0.8) of the missing SNP.
##############################################################################
   ")
  }


  outcome<-give_class(outcome)


  if(file.exists( paste0(file.path( tools::R_user_dir("Oneclick", which = "data") , "path.Rdata" )))){
    load(paste0(file.path( tools::R_user_dir("Oneclick", which = "data") , "path.Rdata" )))
  }else{
    stop("请用U1_set_file_path功能设置附加文件夹")
  }


  if(bfile==""){
    bfile <- file.path(path,"bfile")
  }

  if(plink_bin==""){
    os <- Sys.info()["sysname"]
    a <- paste0("plink/plink_", os)
    if (os == "Windows"){ a <- paste0(a, ".exe") }
    plink_bin <- file.path(path,a)
    if (!file.exists(plink_bin)) {
      stop("请下载相应系统的plink软件并放置在合适的位置！")
    }
  }

  outcome.dat<- U3_extract_outcome_data(outcome =outcome,
                                        exposure_iv = exposure_iv,
                                        proxies=proxies,
                                        max_retries = max_retries,
                                        pop = pop,
                                        bfile = bfile,
                                        plink_bin = plink_bin )
  if( is.null(outcome.dat)  ){
    message("提取不到任何SNP")
  }
  return( outcome.dat )

}

U3_extract_outcome_data <- function( outcome =outcome,
                                     exposure_iv = exposure_iv,
                                     proxies=FALSE,
                                     max_retries = 10,
                                     pop = "EUR",
                                     bfile = "",
                                     plink_bin = "" ){
  UseMethod( "extract_outcome_data" )
}


extract_outcome_data.openGWAS <-  function( outcome = outcome,
                                            exposure_iv = exposure_iv,
                                           proxies=FALSE,
                                           max_retries = 10,
                                           pop = "EUR",
                                           bfile = "",
                                           plink_bin = "" ){

    message("推断为openGWAS的ID")
    message("提取ID结局数据中......")


    outcomes <- ieugwasr::legacy_ids(unique(outcome))

    snps <- unique(exposure_iv$SNP)


    firstpass <- Oneclick_extract_outcome_data_internal(snps = snps, outcomes=outcomes,
                                         proxies = FALSE,max_retries = max_retries)

    if (proxies) {
      for (i in 1:length(outcomes)) {
        if (is.null(firstpass)) {
          missedsnps <- snps
        }
        else {
          missedsnps <- snps[!snps %in% subset(firstpass,
                                               id.outcome == outcomes[i])$SNP]
        }
        if (length(missedsnps) > 0) {
          message("Finding proxies for ", length(missedsnps),
                  " SNPs in outcome ", outcomes[i])
          temp <- Oneclick_extract_outcome_data_internal(snps = missedsnps,
                                          outcomes=outcomes[i], proxies = TRUE,
                                max_retries = max_retries)
          if (!is.null(temp)) {
            firstpass <- plyr::rbind.fill(firstpass, temp)
          }
        }
      }
    }

    if( !is.null(firstpass)  ){

     message("查询样本量中......")

      firstpass <- retry( operation = TwoSampleMR::add_metadata,
                        args=list(dat= firstpass, cols = c("sample_size", "ncase", "ncontrol")) ,
                        max_retries = max_retries)

      firstpass <- firstpass %>%
        TwoSampleMR::split_outcome()

      if( any( grepl( 'finn', firstpass$id.outcome) ) ){
      firstpass <- firstpass %>%
        dplyr::rowwise() %>%
        dplyr::mutate( samplesize.outcome = ifelse( grep( 'finn', id.outcome), ncase.outcome+ncontrol.outcome,  samplesize.outcome )   )
      }

      if( any( firstpass$outcome == "" ) ){
        suppressMessages( ao<-U1_get_ao() )

        firstpass <- firstpass %>%
          dplyr::rowwise() %>%
          dplyr::mutate( outcome = ifelse( outcome == '', ao$trait[which(ao$id == id.outcome )],outcome    )   )

      }


    }
     return(firstpass)

}

# 第三个功能2
extract_outcome_data.df <- function( outcome =outcome,
                                     exposure_iv = exposure_iv,
                                    proxies=FALSE,
                                    max_retries = 10,
                                    pop = "EUR",
                                    bfile = "",
                                    plink_bin = "" ){

    MissingSNPs <- setdiff(exposure_iv$SNP, intersect(exposure_iv$SNP, outcome$SNP))
    message("推断为数据框")
    if( length(MissingSNPs) < 1){

      outcome.dat<- merge(exposure_iv,outcome) %>%
        TwoSampleMR::format_data(type = "outcome") %>%
        dplyr::distinct( SNP,.keep_all = TRUE)
    }else{
      if(proxies == FALSE){
        outcome.dat<- merge(exposure_iv,outcome) %>%
          TwoSampleMR::format_data(type = "outcome") %>%
          dplyr::distinct( SNP,.keep_all = TRUE)
      }else{


        message("寻找r2>0.8的代理SNP中...... \n")

        if(!bfile==""){
          save(bfile, file = paste0(file.path( tools::R_user_dir("Oneclick", which = "data") ,"bfile" ), ".Rdata" )   )
        }else{
          if(file.exists(paste0(file.path( tools::R_user_dir("Oneclick", which = "data") ,"bfile" ), ".Rdata" ))){
            load(file = paste0(file.path( tools::R_user_dir("Oneclick", which = "data") ,"bfile" ), ".Rdata" ) )
            cat("您的","bfile","目录为",bfile,"\n" )
          }
        }

        if(!plink_bin==""){
          save(plink_bin, file = paste0(file.path( tools::R_user_dir("Oneclick", which = "data") ,"plink_bin" ), ".Rdata" )   )
        }else{
          if(file.exists(paste0(file.path( tools::R_user_dir("Oneclick", which = "data") ,"plink_bin" ), ".Rdata" ))){
            load(file = paste0(file.path( tools::R_user_dir("Oneclick", which = "data") ,"plink_bin" ), ".Rdata" ) )
            cat("您的","plink_bin","目录为",plink_bin,"\n" )
          }
        }


        proxy_snp <- find_proxy_snp(MissingSNPs, pop, bfile, plink_bin )


        if(plyr::empty(proxy_snp)){
          message("没找到可用的代理SNP \n")
          outcome.dat<- merge(exposure_iv,outcome) %>%
            TwoSampleMR::format_data(type = "outcome") %>%
            dplyr::distinct( SNP,.keep_all = TRUE)

        } else if (
          plyr::empty(dplyr::filter(proxy_snp, SNP_A != SNP_B)) |
          plyr::empty(dplyr::filter(outcome, SNP %in% (dplyr::filter(proxy_snp, SNP_A != SNP_B) %>% dplyr::pull(SNP_B)))) ){
          message("没找到可用的代理SNP \n")
          outcome.dat<- merge(exposure_iv,outcome) %>%
            TwoSampleMR::format_data(type = "outcome") %>%
            dplyr::distinct( SNP,.keep_all = TRUE)
        } else {
          message("WRANGLING TARGET AND PROXY SNPs \n")
          ## dplyr::filter Query SNPs
          query_snps <- proxy_snp %>%
            dplyr::filter(SNP_A == SNP_B) %>%
            dplyr::select(CHR_A, BP_A, SNP_A, PHASE)

          ## dplyr::filter Proxy SNPs
          proxy.snps <- proxy_snp %>%
            dplyr::filter(SNP_A != SNP_B) %>%                        ## remove query snps
            dplyr::left_join(outcome, by = c('SNP_B' = 'SNP')) %>%
            dplyr::filter(!is.na(effect_allele)) %>%                 ## remove snps with missing information
            dplyr::group_by(SNP_A) %>%                      ## by query snp
            dplyr::arrange(-R2) %>%                                  ## arrange by ld
            dplyr::slice(1) %>%                                      ## dplyr::select top proxy snp
            dplyr::ungroup() %>%
            dplyr::rename(ALT.proxy = effect_allele,
                          REF.proxy = other_allele) %>%
            dplyr::select(-CHR_A, -CHR_B, -BP_A, -BP_B, -MAF_A, -MAF_B, -DP)            ## remove uneeded columns

          ## dplyr::select correlated alleles
          alleles <- proxy.snps %>% dplyr::select(PHASE) %>%
            dplyr::mutate(PHASE = stringr::str_replace(PHASE, "/", ""))
          alleles <- stringr::str_split(alleles$PHASE, "", n = 4, simplify = T)
          colnames(alleles) <- c('ref', 'ref.proxy', 'alt', 'alt.proxy')
          alleles <- tibble::as_tibble(alleles)

          ## Bind Proxy SNPs and correlated alleles
          proxy.out <- proxy.snps %>%
            dplyr::bind_cols(alleles) %>%
            dplyr::rename(SNP = SNP_A) %>%
            dplyr::mutate(effect_allele = ifelse(ALT.proxy == ref.proxy, ref, alt)) %>%
            dplyr::mutate(other_allele = ifelse(REF.proxy == ref.proxy, ref, alt)) %>%
            dplyr::mutate(chr = as.numeric(chr))


          out <- proxy.out %>%
            dplyr::full_join(dplyr::select(query_snps, SNP_A), by = c('SNP' = 'SNP_A')) %>%
            dplyr::rename(proxy_snp = SNP_B, r2 = R2)


          out1<- format_data( out, type = "outcome" ) %>%
            dplyr::full_join(dplyr::select(out, SNP, proxy_snp, r2 ), by = c('SNP' = 'SNP'))

          out2<- merge(exposure_iv,outcome) %>%
            TwoSampleMR::format_data(type = "outcome") %>%
            dplyr::distinct( SNP,.keep_all = TRUE)

          outcome.dat <- plyr::rbind.fill(out1,out2)  %>%
            dplyr::filter(!is.na(beta.outcome))


        }
      }
    }
    return(outcome.dat)
}


# 第三个功能3
extract_outcome_data.Oneclick <- function(  outcome =outcome,
                                            exposure_iv = exposure_iv,
                                          proxies=FALSE,
                                          max_retries = 10,
                                          pop = "EUR",
                                          bfile = "",
                                          plink_bin = "" ){

    message("推断为parquet文件")

    d <- list()
    pb <- progress::progress_bar$new(total = length(outcome))
    for (i in 1:length(outcome)) {

    outcome.dat <- arrow::read_parquet(outcome[i])
    class(outcome.dat)<-c("df",class(outcome.dat))

    suppressMessages(   outcome.dat <- U3_extract_outcome_data(outcome = outcome.dat,
                                          exposure_iv = exposure_iv,
                                          proxies=proxies,
                                          max_retries = max_retries,
                                          pop = pop,
                                          bfile = bfile,
                                          plink_bin = plink_bin ) )

    d[[i]] <- outcome.dat
    gc()
    pb$tick()

    }

    return(plyr::rbind.fill(d))
}


extract_outcome_data.list<-function( outcome =outcome,
                                     exposure_iv = exposure_iv,
                                     proxies=FALSE,
                                     max_retries = 10,
                                     pop = "EUR",
                                     bfile = "",
                                     plink_bin = ""){

  outs<-c()
  for (i in 1:length(outcome)  ){
    d<- U3_extract_outcome_data( give_class(outcome[[i]]),
                                exposure_iv = exposure_iv,
                                proxies=proxies,
                                max_retries = max_retries,
                                pop = pop,
                                bfile = bfile,
                                plink_bin = plink_bin )
  outs <- plyr::rbind.fill(outs,d)
  }



  return(outs)

}


Oneclick_extract_outcome_data_internal <- function (snps, outcomes, proxies = TRUE, rsq = 0.8, align_alleles = 1,
                                                       palindromes = 1, maf_threshold = 0.3, access_token = ieugwasr::check_access_token(),
                                                       splitsize = 10000,max_retries=10)
    {
      snps <- unique(snps)
      message("Extracting data for ", length(snps), " SNP(s) from ",
              length(unique(outcomes)), " GWAS(s)")
      outcomes <- unique(outcomes)
      if (proxies == FALSE) {
        proxies <- 0
      }else if (proxies == TRUE) {
        proxies <- 1
      }else {
        stop("'proxies' argument should be TRUE or FALSE")
      }
      if ((length(snps) < splitsize & length(outcomes) < splitsize) |
          (length(outcomes) < splitsize & length(snps) < splitsize)) {
        d <- retry(operation = Oneclick_associations,
                   args=list(variants = snps, id = outcomes,
                             proxies = proxies, r2 = rsq, align_alleles = align_alleles,
                             palindromes = palindromes, maf_threshold = maf_threshold,
                             access_token = access_token ),
                   max_retries = max_retries)
        if (!is.data.frame(d))
          d <- data.frame()
      }else if (length(snps) > length(outcomes)) {
        n <- length(snps)
        splits <- data.frame(snps = snps, chunk_id = rep(1:(ceiling(n/splitsize)),
                                                         each = splitsize)[1:n])
        d <- list()
        for (i in 1:length(outcomes)) {
          message(i, " of ", length(outcomes), " outcomes")
          d[[i]] <- plyr::ddply(splits, c("chunk_id"), function(x) {
            x <- plyr::mutate(x)
            message(" [>] ", x$chunk_id[1], " of ", max(splits$chunk_id),
                    " chunks")
            out <- retry(operation = Oneclick_associations,
                         args=list(variants = x$snps,
                                   id = outcomes[i], proxies = proxies, r2 = rsq,
                                   align_alleles = align_alleles, palindromes = palindromes,
                                   maf_threshold = maf_threshold, access_token = access_token),
                         max_retries = max_retries)
            if (!is.data.frame(out))
              out <- data.frame()
            return(out)
          })
        }
        d <- plyr::rbind.fill(d)
      }else {
        n <- length(outcomes)
        splits <- data.frame(outcomes = outcomes, chunk_id = rep(1:(ceiling(n/splitsize)),
                                                                 each = splitsize)[1:n])
        d <- list()
        for (i in 1:length(snps)) {
          message(i, " of ", length(snps), " snps")
          d[[i]] <- plyr::ddply(splits, c("chunk_id"), function(x) {
            x <- plyr::mutate(x)
            message(" [>] ", x$chunk_id[1], " of ", max(splits$chunk_id),
                    " chunks")
            out <- retry(operation = Oneclick_associations,
                         args=list(variants = snps[i],
                                   id = x$outcomes, proxies = proxies, r2 = rsq,
                                   align_alleles = align_alleles, palindromes = palindromes,
                                   maf_threshold = maf_threshold, access_token = access_token),
                         max_retries = max_retries)
            if (!is.data.frame(out))
              out <- data.frame()
            return(out)
          })
        }
        d <- plyr::rbind.fill(d)
      }
      if (is.null(nrow(d)) | nrow(d) == 0) {
        return(NULL)
      }
      d <-TwoSampleMR:::format_d(d)
      if (nrow(d) > 0) {
        d$data_source.outcome <- "igd"
        return(d)
      }
      else {
        return(NULL)
      }
    }




Oneclick_associations <- function(variants, id, proxies = 1, r2 = 0.8, align_alleles = 1,
                                      palindromes = 1, maf_threshold = 0.3, access_token = ieugwasr::check_access_token()){

      id <- ieugwasr:::legacy_ids(id)
      out1 <- ieugwasr:::api_query("associations", query = list(variant = variants,
                                                                id = id, proxies = proxies, r2 = r2, align_alleles = align_alleles,
                                                                palindromes = palindromes, maf_threshold = maf_threshold),
                                   access_token = access_token)
      stopifnot(!out1[["status_code"]]=="502")
      out <- out1   %>% ieugwasr::get_query_content()
      if (class(out) == "response") {
        return(out)
      }else if (is.data.frame(out)) {
        out %>% dplyr::as_tibble() %>% ieugwasr:::fix_n() %>% return()
      }else {
        return(dplyr::tibble())
      }

    }


find_proxy_snp<-function(MissingSNPs,pop, bfile, plink_bin ){

      shell <- ifelse(Sys.info()["sysname"] == "Windows", "cmd",
                      "sh")

      write(MissingSNPs,file ="MissingSNPs.txt")

      bfile = file.path( bfile, pop)

      fun <- paste0(shQuote(plink_bin, type = shell),
                    " --bfile ",shQuote(bfile, type = shell),
                    " --keep-allele-order ",
                    " --r2 dprime in-phase with-freqs ",
                    " --ld-snp-list MissingSNPs.txt",
                    " --ld-window-r2 0.8 --ld-window-kb 500 --ld-window 1000 ",
                    " --out Proxys")

      system(fun)

      res <- data.table::fread("Proxys.ld")
      unlink("Proxys.ld")
      unlink("MissingSNPs.txt")

      return(res)

    }




